import requests
import os
from datetime import datetime
import json
import gzip
import talib
import html5lib
import re
import StringIO
from pyquery import PyQuery as pq
import traceback
import numpy as np
import cPickle

proxies = {
    "http": "http://192.168.47.1:8580",
    "https": "http://192.168.47.1:8580",
}
symbol = 'FXUSDJPY'+'FXBTOL20'
def readDataWS():
    endtime=datetime.now().strftime('%Y-%m-%d %H:%M:%S');
    res=requests.get("http://db2015.wstock.cn/wsDB_API2/kline.php?symbol=FXUSDJPY&return_t=3&qt_type=15&r_type=2&"+
                     "stime=1970-01-01 00:00:00&etime=%s&u=test&p=8e6a"%endtime)
    if res.status_code==200:
        data=res.json()
        datalist=[]
        for line in reversed(data):
            datalist.append({
                "Date":line["Date"],
                "Open":float(line["Open"]),
                "High":float(line["High"]),
                "Low":float(line["Low"]),
                "Close":float(line["Close"]),
            })
        return datalist

def AddDIFF(datalist):
    for one in datalist:
        one["DIFF"]=one["Close"]-one["Open"]
def AddDI(datalist):
    for one in datalist:
        one["DI"]=(one["Close"]*2+one["Low"]+one["High"])/4
def AddWMA(datalist):
    dis=np.array([one["DI"] for one in datalist])
    WMA = talib.MA(dis,30,matype=1)
    for i in xrange(len(datalist)):
        datalist[i]["wma"]=WMA[i]
def AddMACD(datalist):
    dis=np.array([one["DI"] for one in datalist])
    macd, macdsignal, macdhist=talib.MACDEXT(dis, fastperiod=12, fastmatype=1, slowperiod=26, slowmatype=1, signalperiod=9, signalmatype=1)
    for i in xrange(len(datalist)):
        datalist[i]["macd"]=macd[i],macdsignal[i],macdhist[i]
def AddBBands(datalist):
    dis=np.array([one["DI"] for one in datalist])
    upperband, middleband, lowerband = talib.BBANDS(dis, timeperiod=15, nbdevup=2, nbdevdn=2, matype=1)
    for i in xrange(len(datalist)):
        datalist[i]["bband"]=upperband[i], middleband[i], lowerband[i]
def AddRSI(datalist):
    dis=np.array([one["DI"] for one in datalist])
    res=talib.RSI(dis,14)
    for i in xrange(len(datalist)):
        datalist[i]["rsi"]=res[i]
def AddSTOCH(datalist):
    high=np.array([one["High"] for one in datalist])
    low=np.array([one["Low"] for one in datalist])
    close=np.array([one["Close"] for one in datalist])
    slowk, slowd = talib.STOCH(high, low, close, fastk_period=5, slowk_period=3, slowk_matype=1, slowd_period=3, slowd_matype=1)
    for i in xrange(len(datalist)):
        datalist[i]["stoch"]=slowk[i], slowd[i]
def AddSAR(datalist):
    high=np.array([one["High"] for one in datalist])
    low=np.array([one["Low"] for one in datalist])
    optInAcceleration = .02
    optInMaximum = .20
    SAR = talib.SAR(high, low, optInAcceleration, optInMaximum)
    for i in xrange(len(datalist)):
        datalist[i]["sar"]=SAR[i]
def AddCCI(datalist):
    high=np.array([one["High"] for one in datalist])
    low=np.array([one["Low"] for one in datalist])
    close=np.array([one["Close"] for one in datalist])
    ccires=talib.CCI(high,low,close)
    for i in xrange(len(datalist)):
        datalist[i]["cci"]=ccires[i]
def AddROC(datalist):
    close=np.array([one["Close"] for one in datalist])
    rs=talib.ROC(close)
    for i in xrange(len(datalist)):
        datalist[i]["roc"]=rs[i]
def AddWILLR(datalist):
    high=np.array([one["High"] for one in datalist])
    low=np.array([one["Low"] for one in datalist])
    close=np.array([one["Close"] for one in datalist])
    res=talib.WILLR(high,low,close)
    for i in xrange(len(datalist)):
        datalist[i]["wr"]=res[i]
def AddDMI(datalist):
    high=np.array([one["High"] for one in datalist])
    low=np.array([one["Low"] for one in datalist])
    close=np.array([one["Close"] for one in datalist])
    res0=talib.ADX(high,low,close)
    res1=talib.ADXR(high,low,close)
    for i in xrange(len(datalist)):
        datalist[i]["dmi"]=(res0[i],res1[i])
def AddBOP(datalist):
    open=np.array([one["Open"] for one in datalist])
    high=np.array([one["High"] for one in datalist])
    low=np.array([one["Low"] for one in datalist])
    close=np.array([one["Close"] for one in datalist])
    bop=talib.BOP(open,high,low,close)
    for i in xrange(len(datalist)):
        datalist[i]["bop"]=bop[i]
def savedata(data,dbfile):
    try:
        os.remove(dbfile)
    except:
        pass
    with gzip.GzipFile(dbfile,"wb") as f:
        json.dump(data,f)
def loaddate(dbfile):
    with gzip.GzipFile(dbfile,"rb") as f:
        return json.load(f)
def fetchdata():
    datalist=readDataWS()
    AddDI(datalist)
    AddDIFF(datalist)
    AddWMA(datalist)
    AddMACD(datalist)
    AddBBands(datalist)
    AddRSI(datalist)
    AddSTOCH(datalist)
    AddSAR(datalist)
    AddCCI(datalist)
    AddROC(datalist)
    AddWILLR(datalist)
    AddDMI(datalist)
    AddBOP(datalist)
    # print json.dumps(datalist,indent=2)
    print len(datalist)
    savedata(datalist,"data/USDJPYdata.gz")
if __name__ == "__main__":
    fetchdata()